From: Taku Kudo <taku@google.com>
Date: Sun, 19 Jun 2022 16:35:11 +0000 (+0900)
Subject: add verbose option
X-Git-Tag: archive/raspbian/0.1.97-3+rpi1^2~17
X-Git-Url: https://dgit.raspbian.org/%22http://www.example.com/cgi/%22/%22http:/www.example.com/cgi/%22?a=commitdiff_plain;h=2e13228c9405e92ffc68768da3c6ca65e20f3daa;p=sentencepiece.git

add verbose option

Signed-off-by: Kentaro Hayashi <kenhys@gmail.com>

Gbp-Pq: Name 0011-add-verbose-option.patch
---

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index 7f19083..5108074 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -45,7 +45,7 @@ jobs:
 
     - name: Test
       working-directory: ${{github.workspace}}/build
-      run: ctest -C Release
+      run: ctest -C Release --output-on-failure
 
     - name: Package
       working-directory: ${{github.workspace}}/build
diff --git a/src/common.h b/src/common.h
index c27c352..ba951d6 100644
--- a/src/common.h
+++ b/src/common.h
@@ -98,15 +98,6 @@ class Die {
  private:
   bool die_;
 };
-
-template <typename T>
-T &&CheckNotNull(const char *file, int line, const char *exprtext, T &&t) {
-  if (t == nullptr) {
-    std::cerr << file << "(" << line << ") " << exprtext;
-    Abort();
-  }
-  return std::forward<T>(t);
-}
 }  // namespace error
 
 namespace logging {
@@ -158,10 +149,6 @@ inline const char *BaseName(const char *path) {
 #define CHECK_LE(a, b) CHECK((a) <= (b))
 #define CHECK_GT(a, b) CHECK((a) > (b))
 #define CHECK_LT(a, b) CHECK((a) < (b))
-#define CHECK_NOTNULL(val)                                    \
-  ::sentencepiece::error::CheckNotNull(                       \
-      ::sentencepiece::logging::BaseName(__FILE__), __LINE__, \
-      "'" #val "' Must be non NULL", (val))
 
 #define FRIEND_TEST(a, b) friend class a##_Test_##b;
 
diff --git a/src/normalizer.cc b/src/normalizer.cc
index d87f89b..2ab8084 100644
--- a/src/normalizer.cc
+++ b/src/normalizer.cc
@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.!
 
+#include "normalizer.h"
+
 #include <utility>
 #include <vector>
 
 #include "common.h"
-#include "normalizer.h"
 #include "third_party/absl/memory/memory.h"
 #include "third_party/absl/strings/match.h"
 #include "third_party/absl/strings/string_view.h"
@@ -46,9 +47,7 @@ Normalizer::~Normalizer() {}
 
 void Normalizer::Init() {
   absl::string_view index = spec_->precompiled_charsmap();
-  if (index.empty()) {
-    LOG(INFO) << "precompiled_charsmap is empty. use identity normalization.";
-  } else {
+  if (!index.empty()) {
     absl::string_view trie_blob, normalized;
 #ifdef IS_BIG_ENDIAN
     status_ = DecodePrecompiledCharsMap(index, &trie_blob, &normalized,
diff --git a/src/sentencepiece_processor.cc b/src/sentencepiece_processor.cc
index a6f5395..805e0f9 100644
--- a/src/sentencepiece_processor.cc
+++ b/src/sentencepiece_processor.cc
@@ -67,12 +67,12 @@ ImmutableSentencePieceText::ImmutableSentencePiece::ImmutableSentencePiece(
     const SentencePieceText_SentencePiece &sp)
     : sp_(&sp) {}
 
-absl::string_view ImmutableSentencePieceText::ImmutableSentencePiece::piece()
+const std::string &ImmutableSentencePieceText::ImmutableSentencePiece::piece()
     const {
   return sp_->piece();
 }
 
-absl::string_view ImmutableSentencePieceText::ImmutableSentencePiece::surface()
+const std::string &ImmutableSentencePieceText::ImmutableSentencePiece::surface()
     const {
   return sp_->surface();
 }
@@ -109,8 +109,10 @@ ImmutableSentencePieceText::pieces(int index) const {
       spt_->pieces(index));
 }
 
-absl::string_view ImmutableSentencePieceText::text() const {
-  return spt_ ? spt_->text() : "";
+const std::string &ImmutableSentencePieceText::text() const {
+  if (spt_) return spt_->text();
+  static std::string *kEmptyString = new std::string();
+  return *kEmptyString;
 }
 
 float ImmutableSentencePieceText::score() const {
diff --git a/src/sentencepiece_processor.h b/src/sentencepiece_processor.h
index 51c5b3b..8124c59 100644
--- a/src/sentencepiece_processor.h
+++ b/src/sentencepiece_processor.h
@@ -165,8 +165,8 @@ class ImmutableSentencePieceText {
   class ImmutableSentencePiece {
    public:
     ~ImmutableSentencePiece() = default;
-    absl::string_view piece() const;
-    absl::string_view surface() const;
+    const std::string &piece() const;
+    const std::string &surface() const;
     uint32_t id() const;
     uint32_t begin() const;
     uint32_t end() const;
@@ -182,7 +182,7 @@ class ImmutableSentencePieceText {
   std::vector<ImmutableSentencePiece> pieces() const;
   size_t pieces_size() const;
   ImmutableSentencePiece pieces(int index) const;
-  absl::string_view text() const;
+  const std::string &text() const;
   float score() const;
 
   std::string SerializeAsString() const;
@@ -193,7 +193,6 @@ class ImmutableSentencePieceText {
   SentencePieceText *mutable_proto();
 
   friend class ImmutableNBestSentencePieceText;
-  friend class SentencePieceProcessor;
 
  private:
   explicit ImmutableSentencePieceText(const SentencePieceText &spt);
@@ -222,8 +221,6 @@ class ImmutableNBestSentencePieceText {
   // it returns the raw pointer managed by the shared_ptr.
   NBestSentencePieceText *mutable_proto();
 
-  friend class SentencePieceProcessor;
-
  private:
   std::shared_ptr<NBestSentencePieceText> rep_;
 };
diff --git a/src/util.h b/src/util.h
index fb312f1..01a561f 100644
--- a/src/util.h
+++ b/src/util.h
@@ -94,7 +94,6 @@ inline bool lexical_cast(absl::string_view arg, std::string *result) {
 
 template <typename T>
 inline bool DecodePOD(absl::string_view str, T *result) {
-  CHECK_NOTNULL(result);
   if (sizeof(*result) != str.size()) {
     return false;
   }